package util;

import com.google.common.collect.Lists;
import org.apache.http.HttpHeaders;
import org.apache.http.message.BasicHeader;
import org.jsoup.select.Elements;
import page.Page;
import page.PageParserTool;
import page.RequestAndResponseTool;

import java.io.*;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.Date;
import java.util.List;
import java.util.Random;

/**
 * @Auther: 黄梓烁
 * @Date: 2020/8/16 0016 19:53
 * @Description:
 */
public class DownloadPdf {

    public void loadInfoFromDBAndDownload() {
        Connection connection = JDBCUtil.getConnection();
        Statement statement = null;
        try {
            statement = connection.createStatement();
            ResultSet resultSet = statement.executeQuery("select id, url from pdfDownload where done = 0");
            while (resultSet.next()) {
                String id = resultSet.getString(1);
                String url = resultSet.getString(2);
                String[] split = url.split("/");
                Statement statement1 = connection.createStatement();
                if(download(url, split[split.length-1] + ".pdf")) {
                    statement1.executeUpdate("update pdfDownload set done = 1 where id = " + id);
                } else {
                    statement1.executeUpdate("update pdfDownload set done = 2 where id = " + id);
                }
                int num = new Random().nextInt(10);
                System.out.println(new Date().toString());
                System.out.println("sleep " + num + " min==================================================");
                Thread.sleep(num*60*1000);
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    /**
     * 本地磁盘放了一个txt文件，存放着未下载文件的文件码。根据文件码来下载文件
     */
    public void loadFileCodeAndDownloadPdf() {
        final String urlPattern = "https://www.sciencedirect.com/science/article/pii/%s";
        final String pathPattern = "D:\\\\pdfpaper\\\\ai\\\\%s\\.pdf";
        try(BufferedReader br = new BufferedReader(new InputStreamReader(
                new FileInputStream("D:\\20201223sciencedirect.txt")))) {
            for (String line = br.readLine(); line != null; line = br.readLine()) {
                String fileName = line + ".pdf";
                if (!new File(String.format(pathPattern, line)).exists()) {
                    download(String.format(urlPattern, line), fileName);
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public boolean download(String url, String filename) {
        boolean isSuccessful = false;
        try {
            List<BasicHeader> headerList = Lists.newArrayList();
            headerList.add(new BasicHeader(HttpHeaders.HOST,"www.sciencedirect.com"));
            headerList.add(new BasicHeader(HttpHeaders.USER_AGENT,"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"));
            headerList.add(new BasicHeader(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"));
            headerList.add(new BasicHeader(HttpHeaders.ACCEPT_LANGUAGE,"zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3"));
            headerList.add(new BasicHeader(HttpHeaders.ACCEPT_ENCODING,"gzip, deflate, br"));
            Page page = RequestAndResponseTool.sendRequstAndGetResponse(headerList,url,null);

            Elements es = PageParserTool.select(page,"div[class=u-margin-s-ver]").select("a[class=anchor]");
            String newUrl = es.attr("href");
            if ("".equals(newUrl)) {
                String html = page.getHtml();
                int i = html.indexOf("\"linkToPdf\":\"");
                if (i == -1) {
                    return isSuccessful;
                }
                int j = html.indexOf("\"", i+13);
                newUrl = html.substring(i+13 , j);
            }

            page = RequestAndResponseTool.sendRequstAndGetResponse("https://www.sciencedirect.com" + newUrl);
            es = PageParserTool.select(page,"div[id=redirect-message]").select("a");
            String redirectUrl = es.attr("href");
            Thread.sleep(new Random().nextInt(1000) + 500);
            downLoadByUrl(redirectUrl, filename);
            isSuccessful = true;
        } catch (Exception e) {
//            e.printStackTrace();
        }

        return isSuccessful;
    }

    public static void  downLoadByUrl(String urlStr,String fileName) throws IOException {
        URL url = new URL(urlStr);
        HttpURLConnection conn = (HttpURLConnection)url.openConnection();
        //设置超时间为3秒
        conn.setConnectTimeout(5*1000);
        //防止屏蔽程序抓取而返回403错误
        conn.setRequestProperty("User-Agent", "Mozilla/4.0 (compatible; MSIE 5.0; Windows NT; DigExt)");
        //得到输入流
        InputStream inputStream = conn.getInputStream();
        //获取自己数组
        byte[] getData = readInputStream(inputStream);
        //文件保存位置
        File saveDir = new File("D:\\pdfpaper\\ai");
        if(!saveDir.exists()){
            saveDir.mkdir();
        }
        File file = new File(saveDir+File.separator + fileName);
        FileOutputStream fos = new FileOutputStream(file);
        fos.write(getData);
        if(fos!=null){
            fos.close();
        }
        if(inputStream!=null){
            inputStream.close();
        }

    }

    public static  byte[] readInputStream(InputStream inputStream) throws IOException {
        byte[] buffer = new byte[1024];
        int len = 0;
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        while((len = inputStream.read(buffer)) != -1) {
            bos.write(buffer, 0, len);
        }
        bos.close();
        return bos.toByteArray();
    }

    /**
     * 以前下载医学文章用过的函数
     * @param fileUrl
     */
    public static void downloadPdf(String fileUrl) {
        if (fileUrl == null) {
            fileUrl = "https://www.biorxiv.org/content/10.1101/2020.05.03.074914v1.full.pdf";
        }
        try {
            URL url = new URL(fileUrl);
            HttpURLConnection urlCon = (HttpURLConnection) url.openConnection();
            urlCon.setConnectTimeout(6000);
            urlCon.setReadTimeout(6000);
            int code = urlCon.getResponseCode();
            if (code != HttpURLConnection.HTTP_OK) {
                throw new Exception("文件读取失败");
            }
            //读文件流
            DataInputStream in = new DataInputStream(urlCon.getInputStream());
            DataOutputStream out = new DataOutputStream(new FileOutputStream("D://test.pdf"));
            byte[] buffer = new byte[2048];
            int count = 0;
            while ((count = in.read(buffer)) > 0) {
                out.write(buffer, 0, count);
            }
            if(out != null) {
                out.close();
            }
            if(in != null) {
                in.close();
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    public static void main(String[] args) {
        new DownloadPdf().loadFileCodeAndDownloadPdf();
    }

}
